import scrapy
from lxml import etree
import json
from pymongo import MongoClient
# from ..items import TutorialItem
import  random
import hashlib

class DetailSpider(scrapy.Spider):
    name ="det"


    def start_requests(self):
        # scrapy.log.start()
        urls =[
             'http://www.shicimingju.com/shicimark/wanyueshi.html',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse,meta={'url': url})
    def parse(self, response):

        url = response.meta['url']
        print(url)
        pagination = response.css('.pagination a::attr(href)').getall()
        item = response.css('.layui-col-sm7 div')
        if pagination is not None:

            for i in item:
                href = i.css('h3 a::attr(href)').getall()

                # if href is not None:
                for hr in href :
                    newHref = 'http://www.shicimingju.com' + str(hr)
                    yield scrapy.Request(url=newHref, callback=self.getDetail, meta={'url': newHref})

            plen = (len(pagination)) - 1
            newHref = 'http://www.shicimingju.com' + str(pagination[plen])
            yield scrapy.Request(url=newHref, callback=self.parse,meta={'url': newHref})

        else :
            for i in item:
             href = i.css('h3 a::attr(href)').getall()

             if href is not None:
                newHref = 'http://www.shicimingju.com' + str(href)
                yield scrapy.Request(url=newHref, callback=self.getDetail,meta={'url': newHref})


    def getDetail(self,response):
        url = response.meta['url']
        print("详情"+url)
        title = response.css('.shici-title::text').extract_first()
        # print(title)

        niandai = response.css('.shici-info::text').extract_first().strip()
        shiren = response.css('.shici-info a::text').extract_first().strip()
        content =  (response.css('.shici-content::text').getall())
        newContent=""
        # item = TutorialItem()

        for cj in content :
            newContent+=cj

        item['_id'] = title + str(random.randint(1, 1000))
        item["title"]=title
        item["niandai"]=niandai
        item["shiren"]=shiren
        item["content"]=newContent
        # print(item)

        # yield item





